# Install 7z
!apt-get install -y p7zip
# Extract datasets
!7z x -y MAXZEGA.7z
!7z x -y IZ0CVY.7z
!7z x -y SV2HWM.7z
#!7z x -y IU8QAI.7z
### Merge all datasets in a huge one
## Currently not used
##!if [ ! -f MERGED.csv ]; then cp MAXZEGA.csv MERGED.csv; tail -n +2 IZ0CVY.csv >> MERGED.csv; tail -n +2 SV2HWM.csv >> MERGED.csv; fi
Reading package lists... Done
Building dependency tree
Reading state information... Done
p7zip is already the newest version (16.02+dfsg-6).
0 upgraded, 0 newly installed, 0 to remove and 39 not upgraded.
7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)
Scanning the drive for archives:
0M Sca 1 file, 95857562 bytes (92 MiB)
Extracting archive: MAXZEGA.7z
--
Path = MAXZEGA.7z
Type = 7z
Physical Size = 95857562
Headers Size = 130
Method = BZip2
Solid = -
Blocks = 1
2% - MAXZEGA.c 4% - MAXZEGA.c 6% - MAXZEGA.c 8% - MAXZEGA.c 11% - MAXZEGA.c 13% - MAXZEGA.c 15% - MAXZEGA.c 17% - MAXZEGA.c 20% - MAXZEGA.c 22% - MAXZEGA.c 25% - MAXZEGA.c 27% - MAXZEGA.c 29% - MAXZEGA.c 31% - MAXZEGA.c 33% - MAXZEGA.c 35% - MAXZEGA.c 37% - MAXZEGA.c 39% - MAXZEGA.c 41% - MAXZEGA.c 43% - MAXZEGA.c 45% - MAXZEGA.c 47% - MAXZEGA.c 50% - MAXZEGA.c 52% - MAXZEGA.c 54% - MAXZEGA.c 56% - MAXZEGA.c 58% - MAXZEGA.c 60% - MAXZEGA.c 63% - MAXZEGA.c 65% - MAXZEGA.c 67% - MAXZEGA.c 69% - MAXZEGA.c 71% - MAXZEGA.c 73% - MAXZEGA.c 76% - MAXZEGA.c 78% - MAXZEGA.c 80% - MAXZEGA.c 82% - MAXZEGA.c 85% - MAXZEGA.c 87% - MAXZEGA.c 89% - MAXZEGA.c 91% - MAXZEGA.c 93% - MAXZEGA.c 95% - MAXZEGA.c 97% - MAXZEGA.c Everything is Ok
Size: 418660575
Compressed: 95857562
7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)
Scanning the drive for archives:
0M Sca 1 file, 73478268 bytes (71 MiB)
Extracting archive: IZ0CVY.7z
--
Path = IZ0CVY.7z
Type = 7z
Physical Size = 73478268
Headers Size = 130
Method = LZMA2:24
Solid = -
Blocks = 1
1% - IZ0CVY.cs 4% - IZ0CVY.cs 6% - IZ0CVY.cs 8% - IZ0CVY.cs 10% - IZ0CVY.cs 12% - IZ0CVY.cs 14% - IZ0CVY.cs 17% - IZ0CVY.cs 19% - IZ0CVY.cs 21% - IZ0CVY.cs 24% - IZ0CVY.cs 26% - IZ0CVY.cs 28% - IZ0CVY.cs 31% - IZ0CVY.cs 33% - IZ0CVY.cs 36% - IZ0CVY.cs 38% - IZ0CVY.cs 40% - IZ0CVY.cs 42% - IZ0CVY.cs 44% - IZ0CVY.cs 47% - IZ0CVY.cs 49% - IZ0CVY.cs 51% - IZ0CVY.cs 54% - IZ0CVY.cs 57% - IZ0CVY.cs 59% - IZ0CVY.cs 62% - IZ0CVY.cs 65% - IZ0CVY.cs 67% - IZ0CVY.cs 70% - IZ0CVY.cs 72% - IZ0CVY.cs 75% - IZ0CVY.cs 77% - IZ0CVY.cs 79% - IZ0CVY.cs 81% - IZ0CVY.cs 84% - IZ0CVY.cs 86% - IZ0CVY.cs 89% - IZ0CVY.cs 92% - IZ0CVY.cs 94% - IZ0CVY.cs 95% - IZ0CVY.cs 98% - IZ0CVY.cs 100% Everything is Ok
Size: 477379756
Compressed: 73478268
7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21
p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)
Scanning the drive for archives:
0M Sca 1 file, 22450501 bytes (22 MiB)
Extracting archive: SV2HWM.7z
--
Path = SV2HWM.7z
Type = 7z
Physical Size = 22450501
Headers Size = 130
Method = BZip2
Solid = -
Blocks = 1
7% - SV2HWM.cs 17% - SV2HWM.cs 23% - SV2HWM.cs 32% - SV2HWM.cs 39% - SV2HWM.cs 49% - SV2HWM.cs 54% - SV2HWM.cs 61% - SV2HWM.cs 67% - SV2HWM.cs 72% - SV2HWM.cs 76% - SV2HWM.cs 81% - SV2HWM.cs 89% - SV2HWM.cs 96% - SV2HWM.cs Everything is Ok
Size: 103557518
Compressed: 22450501
# Set the station callsign
CALLSIGN = "IZ0CVY"
# Import libraries
import numpy as np
import seaborn as sn
import matplotlib.pyplot as plt
import pandas as pd
from matplotlib.pyplot import figure
from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits import mplot3d
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
# df = pd.read_csv('/content/' + CALLSIGN + '.csv', sep=';')
CURPATH = "/content"
CURPATH = "/home/jovyan/work/logs/IZ0CVY/fullogs"
df = pd.read_csv(CURPATH + "/" + CALLSIGN + '.csv', sep=';')
# Ignore strong signals to avoid biases
df = df.query("(snr > 0) | (snr < 70)")
# Remove unused (test) columns
df = df.drop('tx_pow', axis='columns')
df = df.drop('rec_pow', axis='columns')
df.head(10)
| latitude | longitude | altitude | elevation | azimuth | distance | snr | vel_v | vel_h | heading | temp | humidity | burst_timer | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 41.04255 | 13.64955 | 6870.9 | 1.44 | 142.99 | 174.0 | 8.4 | -7.0 | 31.7 | 127.3 | -273.0 | -1.0 | -1.0 |
| 1 | 41.04238 | 13.64985 | 6864.4 | 1.44 | 142.98 | 174.0 | 7.5 | -6.8 | 31.3 | 126.1 | -273.0 | -1.0 | -1.0 |
| 2 | 41.04222 | 13.65014 | 6857.8 | 1.43 | 142.98 | 174.1 | 7.5 | -6.5 | 30.2 | 125.5 | -273.0 | -1.0 | -1.0 |
| 3 | 41.04206 | 13.65043 | 6851.1 | 1.43 | 142.98 | 174.1 | 7.7 | -7.1 | 29.6 | 126.7 | -273.0 | -1.0 | -1.0 |
| 4 | 41.04173 | 13.65098 | 6837.8 | 1.43 | 142.97 | 174.2 | 7.3 | -7.0 | 29.5 | 129.0 | -273.0 | -1.0 | -1.0 |
| 5 | 41.67285 | 12.47723 | 1615.7 | 0.91 | 175.08 | 69.8 | 13.7 | 5.6 | 14.4 | 102.1 | -273.0 | -1.0 | -1.0 |
| 6 | 41.67282 | 12.47738 | 1620.8 | 0.91 | 175.07 | 69.8 | 12.8 | 5.8 | 11.8 | 105.5 | -273.0 | -1.0 | -1.0 |
| 7 | 41.67280 | 12.47751 | 1626.6 | 0.91 | 175.06 | 69.8 | 13.9 | 6.4 | 10.0 | 100.5 | -273.0 | -1.0 | -1.0 |
| 8 | 41.67279 | 12.47762 | 1632.5 | 0.92 | 175.05 | 69.8 | 13.9 | 5.4 | 9.4 | 88.5 | -273.0 | -1.0 | -1.0 |
| 9 | 41.67281 | 12.47774 | 1636.8 | 0.92 | 175.05 | 69.8 | 12.5 | 4.7 | 9.5 | 74.7 | -273.0 | -1.0 | -1.0 |
df.describe()
| latitude | longitude | altitude | elevation | azimuth | distance | snr | vel_v | vel_h | heading | temp | humidity | burst_timer | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 |
| mean | 4.177035e+01 | 1.295481e+01 | 1.781598e+04 | 9.696972e+00 | 1.511424e+02 | 1.176827e+02 | 1.896516e+01 | 2.138200e-01 | 1.659990e+01 | 1.406110e+02 | -4.730668e+01 | 1.271648e+01 | 5.971030e-01 |
| std | 8.011265e-01 | 8.655893e-01 | 9.479105e+03 | 7.366231e+00 | 4.700365e+01 | 7.153582e+01 | 4.565435e+00 | 1.037528e+01 | 1.158022e+01 | 8.141628e+01 | 2.955356e+01 | 2.170625e+01 | 1.494908e+01 |
| min | 3.792109e+01 | 8.449580e+00 | 3.800000e+02 | 1.100000e-01 | 0.000000e+00 | 8.500000e+00 | 2.800000e+00 | -1.355000e+02 | 0.000000e+00 | 0.000000e+00 | -2.730000e+02 | -1.000000e+00 | -1.000000e+00 |
| 25% | 4.146011e+01 | 1.248843e+01 | 9.755700e+03 | 4.190000e+00 | 1.313300e+02 | 7.420000e+01 | 1.600000e+01 | -4.000000e-01 | 8.300000e+00 | 8.280000e+01 | -5.840000e+01 | 1.100000e+00 | -1.000000e+00 |
| 50% | 4.165167e+01 | 1.279676e+01 | 1.749200e+04 | 8.290000e+00 | 1.564900e+02 | 9.710000e+01 | 1.980000e+01 | 4.500000e+00 | 1.390000e+01 | 1.125000e+02 | -5.320000e+01 | 1.800000e+00 | -1.000000e+00 |
| 75% | 4.184738e+01 | 1.323182e+01 | 2.551540e+04 | 1.314000e+01 | 1.748500e+02 | 1.290000e+02 | 2.240000e+01 | 5.500000e+00 | 2.190000e+01 | 1.875000e+02 | -3.860000e+01 | 1.340000e+01 | -1.000000e+00 |
| max | 4.622024e+01 | 1.850719e+01 | 3.841760e+04 | 8.118000e+01 | 3.600000e+02 | 5.562000e+02 | 3.720000e+01 | 8.880000e+01 | 1.088000e+02 | 3.600000e+02 | 3.160000e+01 | 1.000000e+02 | 8.471000e+02 |
ddf = df
fig = plt.figure(figsize=(16, 16))
fig.set_dpi(90)
# AZIMUTH vs SNR
ax1 = plt.subplot(1,3,1, projection = 'polar')
ax1.set_theta_zero_location("N")
ax1.scatter(-np.deg2rad(ddf.azimuth), ddf.snr, s=0.2, cmap='jet')
plt.ylim(0, 45)
# ax1.set_xlabel('azimuth', fontsize = 20, labelpad = 20)
ax1.set_title('Azimuth vs SNR')
# ELEVATION vs SNR
ax2 = fig.add_subplot(1,3,2, projection='polar', xlim=(0, 90))
ax2.set_theta_zero_location("N")
ax2.set_thetamin(-90) # set the limits
ax2.set_thetamax(90)
ax2.set_theta_offset(np.deg2rad(0)) # point the origin towards the top
plt.ylim(0, 35)
# ax2.set_thetagrids(range(-90, 120, 30)) # set the gridlines
ax2.scatter(np.deg2rad(ddf.elevation), ddf.snr, s=0.2, cmap='jet')
# plt.ylim(0, 45)
ax2.set_title('Elevation vs SNR')
# AZIMUTH vs ELEVATION with SNR
ax3 = fig.add_subplot(1,3,3, projection='polar')
ax3.set_theta_zero_location("N")
# Create chart
plot = ax3.scatter(-np.deg2rad(ddf['azimuth']), ddf['elevation'], s=0.2, c=ddf['snr'], cmap='jet')
ax3.set_title('Azimuth vs Elevation (with SNR)')
# Create legend
cbaxes = inset_axes(ax3, width="1%", height="30%", loc=1)
plt.colorbar(plot, cax=cbaxes, orientation='vertical', ticks=np.linspace(ddf['snr'].min(), ddf['snr'].max(), 6))
# Show chart
plt.show()
# DISTANCE vs SNR
fig = plt.figure(figsize=(18, 5))
fig.set_dpi(85)
ax3 = fig.add_subplot(1,1,1)
plt.grid()
plt.scatter(ddf.distance, ddf.snr, s=0.1)
plt.xlim(0, 500)
plt.ylim(0, 40)
# ax3.set_xlabel('distance', fontsize = 20, labelpad = 20)
ax3.set_title('Distance vs SNR')
ax3.set_xticks(np.linspace(0,500,11))
plt.show()
# # produce a legend with the unique colors from the scatter
# legend1 = ax.legend(*scatter.legend_elements(),
# loc="lower left", title="Classes")
# ax.add_artist(legend1)
# ax3 = fig.add_subplot(111, projection = '3d')
# # ax3.set_theta_zero_location("N")
# plot = ax3.scatter(ddf.distance, ddf.snr, s=0.2, cmap='jet')
# ax3.set_xlabel('distance', fontsize = 20, labelpad = 20)
# ax3.set_title('Distance vs snr')
# Copy the dataframe, and drop unused columns
ddf = df
ddf = ddf.drop('latitude', axis='columns')
ddf = ddf.drop('longitude', axis='columns')
#ddf = ddf.drop('tx_pow', axis='columns')
#ddf = ddf.drop('rec_pow', axis='columns')
fig = plt.figure(figsize=(16, 16))
fig.set_dpi(75)
# Create the correlation matrix
corr_matrix = ddf.corr(method='spearman')
sn.heatmap(corr_matrix, annot=True)
# Show the correlation matrix
plt.show()
# Copy the dataframe
ddf = df
ddf.describe()
| latitude | longitude | altitude | elevation | azimuth | distance | snr | vel_v | vel_h | heading | temp | humidity | burst_timer | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 | 5.731520e+06 |
| mean | 4.177035e+01 | 1.295481e+01 | 1.781598e+04 | 9.696972e+00 | 1.511424e+02 | 1.176827e+02 | 1.896516e+01 | 2.138200e-01 | 1.659990e+01 | 1.406110e+02 | -4.730668e+01 | 1.271648e+01 | 5.971030e-01 |
| std | 8.011265e-01 | 8.655893e-01 | 9.479105e+03 | 7.366231e+00 | 4.700365e+01 | 7.153582e+01 | 4.565435e+00 | 1.037528e+01 | 1.158022e+01 | 8.141628e+01 | 2.955356e+01 | 2.170625e+01 | 1.494908e+01 |
| min | 3.792109e+01 | 8.449580e+00 | 3.800000e+02 | 1.100000e-01 | 0.000000e+00 | 8.500000e+00 | 2.800000e+00 | -1.355000e+02 | 0.000000e+00 | 0.000000e+00 | -2.730000e+02 | -1.000000e+00 | -1.000000e+00 |
| 25% | 4.146011e+01 | 1.248843e+01 | 9.755700e+03 | 4.190000e+00 | 1.313300e+02 | 7.420000e+01 | 1.600000e+01 | -4.000000e-01 | 8.300000e+00 | 8.280000e+01 | -5.840000e+01 | 1.100000e+00 | -1.000000e+00 |
| 50% | 4.165167e+01 | 1.279676e+01 | 1.749200e+04 | 8.290000e+00 | 1.564900e+02 | 9.710000e+01 | 1.980000e+01 | 4.500000e+00 | 1.390000e+01 | 1.125000e+02 | -5.320000e+01 | 1.800000e+00 | -1.000000e+00 |
| 75% | 4.184738e+01 | 1.323182e+01 | 2.551540e+04 | 1.314000e+01 | 1.748500e+02 | 1.290000e+02 | 2.240000e+01 | 5.500000e+00 | 2.190000e+01 | 1.875000e+02 | -3.860000e+01 | 1.340000e+01 | -1.000000e+00 |
| max | 4.622024e+01 | 1.850719e+01 | 3.841760e+04 | 8.118000e+01 | 3.600000e+02 | 5.562000e+02 | 3.720000e+01 | 8.880000e+01 | 1.088000e+02 | 3.600000e+02 | 3.160000e+01 | 1.000000e+02 | 8.471000e+02 |
# Filter results to examine the lower atmosphere
ddf = df.query('altitude <= 14000')
## when running locally, limit the ram usage to avoid crashes.
#ddf = ddf.head(150000)
cols = ['altitude', 'elevation', 'azimuth', 'distance', 'snr', 'vel_v', 'vel_h', 'heading', 'temp', 'humidity']
sns_plot = sn.pairplot(ddf[cols], height=4.0)
sns_plot.savefig(CURPATH + "/sns_pairplot_" + CALLSIGN + "_14000-.png")
# Filter results to examine the higher atmosphere
ddf = df.query('altitude >= 20000')
## when running locally, limit the ram usage to avoid crashes.
#ddf = ddf.head(150000)
cols = ['altitude', 'elevation', 'azimuth', 'distance', 'snr', 'vel_v', 'vel_h', 'heading', 'temp', 'humidity']
sns_plot = sn.pairplot(ddf[cols], height=4.0)
sns_plot.savefig(CURPATH + "/sns_pairplot_" + CALLSIGN + "_20000+.png")
# Filter results to examine the lower atmosphere
ddf = df
## when running locally, limit the ram usage to avoid crashes.
#ddf = ddf.head(150000)
cols = ['altitude', 'elevation', 'azimuth', 'distance', 'snr', 'vel_v', 'vel_h', 'heading', 'temp', 'humidity']
sns_plot = sn.pairplot(ddf[cols], height=4.0)
sns_plot.savefig(CURPATH + "/sns_pairplot_" + CALLSIGN + "_fullrange.png")